//////////////////////////////////////////
//
// OpenCL host program template
//
//                   
// Created by dongwang@2016.01.10
//
/////////////////////////////////////////
#define CL_HPP_CL_1_2_DEFAULT_BUILD
#define CL_HPP_TARGET_OPENCL_VERSION 120
#define CL_HPP_MINIMUM_OPENCL_VERSION 120
#define CL_HPP_ENABLE_PROGRAM_CONSTRUCTION_FROM_ARRAY_COMPATIBILITY 1
#define CL_USE_DEPRECATED_OPENCL_1_2_APIS

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <CL/opencl.h>

// user defined library
#include "ocl_util.h"
#include "timer.h"

using namespace ocl_util;

//----------- Design Parameters --------------//
// select what platform is used
const char *vendor_name = "Xilinx";
//const char *vendor_name = "AMD";
//const char *vendor_name = "Intel"; // SDK version <= 19.1
//#if defined(SW_EMU) // SDK version >= 19.3
//const char *vendor_name = "Intel(R) FPGA Emulation Platform for OpenCL(TM)";
//#else
//const char *vendor_name = "Intel(R) FPGA SDK for OpenCL(TM)";
//#endif
#define DEVICE_TYPE CL_DEVICE_TYPE_ACCELERATOR
//#define DEVICE_TYPE CL_DEVICE_TYPE_GPU

// Define the binary name
const char *kernel_name = "memAccess";

// problem size
#define DATA_SIZE_USED 1*1024*1024
#define DATA_SIZE      1*1024*1024
#define GLOBAL_SIZE    DATA_SIZE
#define DMA_ALIGNMENT  4096 // for Xilinx FPGA must be 4096
#define VEC_SIZE       16

//------------ Global Functions & Variables ------------//
cl_uint num_devices = 0;
cl_platform_id platform_id = NULL;
cl_context context = NULL;
cl_program program = NULL;
scoped_array<cl_device_id> device;
scoped_array<cl_kernel> kernel;
scoped_array<cl_command_queue> queue;
scoped_array<cl_mem> input_a_buf;
scoped_array<cl_mem> output_b_buf;
scoped_array<float *> input_a;
scoped_array<float *> output_b;
scoped_array<cl_event> kernel_event(num_devices);
scoped_array<cl_event> readevent(num_devices);

void cleanup();


int main(int argc, char** argv)
{
	cl_int status;
	unsigned int num_err;               // number of error results returned
    unsigned int device_ptr;
	
	
	Timer t;                            // Timer used for performance measurement
	
	if (argc != 2){
	printf("Error: wrong commad format, usage:\n");  
	printf("%s <inputfile>\n", argv[0]);
	return EXIT_FAILURE;
	}
	
	printf("********************************\n");
	printf("Memory Accessing Micro Benchmark\n");
	printf("********************************\n");
    printf("Data Size for each device: %d MBytes\n", ((unsigned int)(sizeof(float)*DATA_SIZE)/(1024*1024)));
	
	// Connect to the desired platform
	platform_id = findPlatform(vendor_name);
	if(platform_id == NULL) {
		printf("ERROR: Unable to find the desired OpenCL platform.\n");
		return EXIT_FAILURE;
	}

	// Query the available OpenCL device
	device.reset(getDevices(platform_id, DEVICE_TYPE, &num_devices));
	printf("Platform: %s\n", getPlatformName(platform_id).c_str());
    printf("Totally %d device(s) are found\n", num_devices);
    device_ptr = 0; // only use one device, select the proper idx
    num_devices = 1; // reset the num of device to 1
	//for(unsigned i = 0; i < num_devices; ++i) {
		printf("  Device %d: %s\n", device_ptr, getDeviceName(device[device_ptr]).c_str());
		displayDeviceInfo(device[device_ptr]);
	//}
	
	// Initialize the data set for each device
	input_a.reset(num_devices);
	output_b.reset(num_devices);
	for(unsigned i = 0; i < num_devices; ++i) {

		input_a[i] = (float *)alignedMalloc(sizeof(float)*DATA_SIZE, DMA_ALIGNMENT);
		output_b[i] = (float *)alignedMalloc(sizeof(float)*DATA_SIZE, DMA_ALIGNMENT);
		if(input_a[i] == NULL || output_b[i] == NULL)
		{
			printf("Not enough memory !!!");
			alignedFree(input_a[i]);
			alignedFree(output_b[i]);
			return EXIT_FAILURE;
		}
		
		for(unsigned int j = 0; j < DATA_SIZE; j++) {
			input_a[i][j] = rand_float();
		}
		memset(output_b[i], 0, sizeof(float)*DATA_SIZE);
	}

	// Create the context.
	context = clCreateContext(NULL, num_devices, &device[device_ptr], NULL, NULL, &status);
	checkError(status, "Failed to create context");

	// Create Program Objects
	char *kernel_file_name=argv[1];

	// Create the program for all device. All devices execute the same kernel.
	program = createProgramFromFile(context, (const char *) kernel_file_name, &device[device_ptr], num_devices);
	
	// Create per-device objects.
	queue.reset(num_devices);
	kernel.reset(num_devices);
	input_a_buf.reset(num_devices);
	output_b_buf.reset(num_devices);
	
	for(unsigned i = 0; i < num_devices; ++i) {
		// Command queue
		queue[i] = clCreateCommandQueue(context, device[device_ptr], CL_QUEUE_PROFILING_ENABLE, &status);
		checkError(status, "Failed to create command queue");
		
		// Kernel
		kernel[i] = clCreateKernel(program, kernel_name, &status);
		checkError(status, "Failed to create kernel");
		
		// Input buffers
		input_a_buf[i] = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_USE_HOST_PTR, 
			DATA_SIZE * sizeof(float), input_a[i], &status);
		checkError(status, "Failed to create buffer for input A");
		
		// Output buffers
		output_b_buf[i] = clCreateBuffer(context, CL_MEM_WRITE_ONLY | CL_MEM_USE_HOST_PTR, 
			DATA_SIZE * sizeof(float), output_b[i], &status);
		checkError(status, "Failed to create buffer for output B");

		// Set kernel arguments.
		unsigned argi = 0;
		unsigned data_num = DATA_SIZE/VEC_SIZE;
		
		status = clSetKernelArg(kernel[i], argi++, sizeof(cl_mem), &input_a_buf[i]);
		checkError(status, "Failed to set argument %d", argi - 1);
	
		status = clSetKernelArg(kernel[i], argi++, sizeof(cl_mem), &output_b_buf[i]);
		checkError(status, "Failed to set argument %d", argi - 1);
		
		status = clSetKernelArg(kernel[i], argi++, sizeof(cl_uint), &data_num);
		checkError(status, "Failed to set argument %d", argi - 1);
        
		// Enque input buffers, blocking write is used
		status = clEnqueueWriteBuffer(queue[i], input_a_buf[i], CL_TRUE,
			0, DATA_SIZE_USED * sizeof(float), input_a[i], 0, NULL, &readevent[i]);
        //status = clEnqueueMigrateMemObjects(queue[i], 1, &input_a_buf[i], 0, /* 0 means from host*/
        //                                    0, NULL, &readevent[i]);
		checkError(status, "Failed to transfer input A");
	}
    
	// Wait for reads to finish
	clWaitForEvents(num_devices, readevent);
    
	// Recorde the start time
	t.start();
	
	// Execute the kernel

	for(unsigned i = 0; i < num_devices; ++i) {
        
		status = clEnqueueTask(queue[i], kernel[i], 0, NULL, &kernel_event[i]);
        checkError(status, "Failed to launch kernel");
	}
	
	// Wait for all kernel to finish
	clWaitForEvents(num_devices, &kernel_event[0]);

	//Recorde the end time
	t.stop();
	float time = t.get_time_s();
	
	// Read back the results from the device to verify the output
	//
	for(unsigned i = 0; i < num_devices; ++i) {

		status = clEnqueueReadBuffer(queue[i], output_b_buf[i], CL_FALSE, 
			0, sizeof(float) * DATA_SIZE_USED, output_b[i], 0, NULL, &readevent[i]);  
        //status = clEnqueueMigrateMemObjects(queue[i], 1, &output_b_buf[i], 
        //                                    CL_MIGRATE_MEM_OBJECT_HOST, 0, NULL, &readevent[i]);
		checkError(status, "Failed to set transfer output B");
	}
    
	// Wait for reads to finish
	clWaitForEvents(num_devices, &readevent[0]);

	// Evaluate the performance
	// average bandwidth
	float bw;
	printf("\nPerformance Summary\n");
	printf("Total runtime: %fs \n", time);
	// bandwidth for each device
	for(unsigned i = 0; i < num_devices; ++i) {
		cl_ulong time_ns = getKernelStartEndTime(kernel_event[i]);
		printf("Kernel runtime (device %d): %0.3f ms\n", i, double(time_ns) * 1e-6);
        bw = DATA_SIZE * sizeof(float) / (time_ns * 1e-9 * 1000000.0f);
        printf("Average device2ddr bandwidth (RD or WR): %f MBytes/s\n", bw);
	}
	
	// Validate the results
	num_err = 0;
	for(unsigned i = 0; i < num_devices; ++i) {
		for (unsigned int j = 0; j < DATA_SIZE; j++) 
		if (output_b[i][j] != input_a[i][j]){
			printf("num=%d, input=%f, output=%f\n", j, input_a[i][j], output_b[i][j]);
			num_err++; // count how many error are found
		}
	}
	
	// Release resource
	cleanup();
	
	if(num_err == 0){
		printf("\nTest passed!\n");
		return EXIT_SUCCESS;
	}
	else{
		printf("\nTest failed!\n");
		printf("Summary: '%d/%d' error values!\n", num_err, DATA_SIZE);
		return EXIT_FAILURE;
	}
}



// Release all memory resources here
void cleanup()
{

	// Release the opencl runtime resource allocated
	for(unsigned i = 0; i < num_devices; ++i) {
		if(kernel && kernel[i]) {
			clReleaseKernel(kernel[i]);
		}
		if(queue && queue[i]) {
			clReleaseCommandQueue(queue[i]);
		}
		if(input_a_buf && input_a_buf[i]) {
			clReleaseMemObject(input_a_buf[i]);
		}
		if(output_b_buf && output_b_buf[i]) {
			clReleaseMemObject(output_b_buf[i]);
		}
		if(kernel_event && kernel_event[i]) {
			clReleaseEvent(kernel_event[i]);
		}
		if(readevent && readevent[i]) {
			clReleaseEvent(readevent[i]);
		}
	}
	
	if(program) {
		clReleaseProgram(program);
	}
	if(context) {
		clReleaseContext(context);
	}
	// Release the memory resource allocated
	for(unsigned i = 0; i < num_devices; ++i) {
		alignedFree(input_a[i]);
		alignedFree(output_b[i]);
	}
}
